package infiews.fetch;

import infiews.crawler.Config;
import infiews.crawler.Crawler;
import infiews.crawler.TaskManager;
import infiews.util.Logger;
import infiews.web.Page;
import infiews.web.URLCanonicalizer;
import org.apache.http.*;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.params.ConnManagerParams;
import org.apache.http.conn.params.ConnPerRouteBean;
import org.apache.http.conn.params.ConnRoutePNames;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpParams;
import org.apache.http.params.HttpProtocolParamBean;

import java.io.IOException;
import java.util.Date;

public class PageFetcher { 
	
	private final Logger log = Crawler.log();
	private ThreadSafeClientConnManager connectionManager;
	private Object mutex = PageFetcher.class.toString() + "_MUTEX";
	private DefaultHttpClient httpclient;
	private long startOfPeriod = 0;
	private long lastFetchTime = 0;
	private int processedCount;
	private long politenessDelay = Crawler.getConfig().getInt("fetcher.default_politeness_delay");
	private TaskManager taskManager;
	public static int MAX_DOWNLOAD_SIZE;
	private final boolean show404Pages;

	public PageFetcher(){
		HttpParams params = new BasicHttpParams();
		HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params);
		paramsBean.setVersion(HttpVersion.HTTP_1_1);
		paramsBean.setContentCharset("UTF-8");
		paramsBean.setUseExpectContinue(false);

		Config cfg = Crawler.getConfig();
		
		politenessDelay = cfg.getInt("fetcher.default_politeness_delay");
		MAX_DOWNLOAD_SIZE = cfg.getInt("fetcher.max_download_size");
		show404Pages = cfg.getBoolean("logging.show_404_pages");
		
		params.setParameter("http.useragent", cfg.getString( "fetcher.user_agent" ) );
		params.setIntParameter("http.socket.timeout", cfg.getInt("fetcher.socket_timeout"));
		params.setIntParameter("http.connection.timeout", cfg.getInt("fetcher.connection_timeout"));
		params.setBooleanParameter("http.protocol.handle-redirects",cfg.getBoolean("fetcher.follow_redirects"));

		ConnPerRouteBean connPerRouteBean = new ConnPerRouteBean();
		connPerRouteBean.setDefaultMaxPerRoute(cfg.getInt("fetcher.max_connections_per_host"));
		ConnManagerParams.setMaxConnectionsPerRoute(params, connPerRouteBean);
		ConnManagerParams.setMaxTotalConnections(params,cfg.getInt("fetcher.max_total_connections"));

		SchemeRegistry schemeRegistry = new SchemeRegistry();
		schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80));

		if (cfg.getBoolean("fetcher.crawl_https")) {
			schemeRegistry.register(new Scheme("https", SSLSocketFactory.getSocketFactory(), 443));
		}

		connectionManager = new ThreadSafeClientConnManager(params,schemeRegistry);
		httpclient = new DefaultHttpClient(connectionManager, params);
	}
	
	public long getPolitenessDelay() {
		return politenessDelay;
	}

	public void setPolitenessDelay(long politenessDelay) {
		this.politenessDelay = politenessDelay;
	}

	public int fetch(Page page) {
		String toFetchURL = page.getURL();
		HttpGet get = null;
		HttpEntity entity = null;
		try {
			get = new HttpGet(toFetchURL);
			synchronized (mutex) {
				long now = (new Date()).getTime();
				if (now - startOfPeriod > 10000) {
					log.info("Number of pages fetched per second: "
							+ processedCount / ((now - startOfPeriod) / 1000));
					processedCount = 0;
					startOfPeriod = now;
				}
				processedCount++;

				if (now - lastFetchTime < politenessDelay) {
					Thread.sleep(politenessDelay - (now - lastFetchTime));
				}
				lastFetchTime = (new Date()).getTime();
			}
			HttpResponse response = httpclient.execute(get);
			entity = response.getEntity();

			if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
				if (response.getStatusLine().getStatusCode() != HttpStatus.SC_NOT_FOUND) {
					log.info("Failed: "
							+ response.getStatusLine().toString()
							+ ", while fetching " + toFetchURL);
				} else if (show404Pages) {
					log.info("Not Found: " + toFetchURL
							+ " (Link found in #id: "
							+ page.getId() + ")");
				}
				return response.getStatusLine().getStatusCode();
			}

			String uri = get.getURI().toString();
			if (!uri.equals(toFetchURL)) {
				if (!URLCanonicalizer.getCanonicalURL(uri).equals(toFetchURL)) {
					if( taskManager.hasPage(uri)){
							return PageFetchStatus.RedirectedPageIsSeen;
					}
					page.setURL(uri);
				}
			}
			
			if (entity != null) {
				if( response.containsHeader("Last-Modified")){
					Header[] hList = response.getAllHeaders();
					for( int i=0; i<hList.length; i++ ){
						if( hList[0].getName() == "Last-Modified" ){
							if( hList[i].getValue() == page.getLastModified()  ){
								return PageFetchStatus.PageNotChanged;
							}else{
								page.setLastModified( hList[i].getValue() );
							}
						}
					}
				}
				long size = entity.getContentLength();
				if (size == -1) {
					Header length = response.getLastHeader("Content-Length");
					if (length == null) {
						length = response.getLastHeader("Content-length");
					}
					if (length != null) {
						size = Integer.parseInt(length.getValue());
					} else {
						size = -1;
					}
				}
				if (size > MAX_DOWNLOAD_SIZE) {
					entity.consumeContent();
					return PageFetchStatus.PageTooBig;
				}

				boolean isBinary = false;

				Header type = entity.getContentType();
				if (type != null
						&& type.getValue().toLowerCase().contains("image")) {
					isBinary = true;
				}

				if (page.load(entity.getContent(), (int) size, isBinary)) {
					return PageFetchStatus.OK;
				} else {
					return PageFetchStatus.PageLoadError;
				}
			} else {
				get.abort();
			}
		} catch (IOException e) {
			log.error("Fatal transport error: " + e.getMessage()
					+ " while fetching " + toFetchURL + " (link found in doc #"
					+ page.getParentId() + ")");
			return PageFetchStatus.FatalTransportError;
		} catch (IllegalStateException e) {
			// ignoring exceptions that occur because of not registering https
			// and other schemes
		} catch (Exception e) {
			log.exception(e);
		} finally {
			try {
				if( get != null ){
					get.abort();
				}
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		return PageFetchStatus.UnknownError;
	}

	public void setProxy(String proxyHost, int proxyPort) {
		HttpHost proxy = new HttpHost(proxyHost, proxyPort);
		httpclient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY,
				proxy);
	}

	public void setProxy(String proxyHost, int proxyPort,
			String username, String password) {
		httpclient.getCredentialsProvider().setCredentials(
				new AuthScope(proxyHost, proxyPort),
				new UsernamePasswordCredentials(username, password));
		setProxy(proxyHost, proxyPort);
	}
}
